import re
import logging
from typing import List, Dict, Union
import pandas as pd

logger = logging.getLogger(__name__)

def get_text_statistics(text: str) -> Dict[str, Union[int, float]]:
    """
    获取文本的基本统计信息
    """
    stats = {
        'char_count': len(text),
        'word_count': len(text.split()),
        'has_url': 1 if re.search(r'http[s]?://', text) else 0,
        'has_mention': 1 if '@' in text else 0
    }
    return stats

def calculate_dataset_statistics(df: pd.DataFrame) -> Dict[str, Union[int, float]]:
    """
    计算数据集的统计信息
    """
    stats = {
        'total_comments': len(df),
        'avg_text_length': df['cleaned_text'].str.len().mean(),
        'max_text_length': df['cleaned_text'].str.len().max(),
        'min_text_length': df['cleaned_text'].str.len().min()
    }
    return stats

def export_statistics(stats: Dict[str, Union[int, float]], output_path: str):
    """
    导出统计信息到文件
    """
    try:
        with open(output_path, 'w', encoding='utf-8') as f:
            for key, value in stats.items():
                f.write(f"{key}: {value}\n")
        logger.info(f"统计信息已保存到 {output_path}")
    except Exception as e:
        logger.error(f"保存统计信息失败: {str(e)}")
        raise 